package com.rockbb.common.utilities.internet;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.conn.params.ConnRoutePNames;
import org.apache.http.cookie.Cookie;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.util.EntityUtils;
import org.apache.log4j.Logger;

public class Crawler {
	private static Logger log = Logger.getLogger(Crawler.class);

	/*
	 * Public parameters
	 */
	public static int SO_TIMEOUT = 1000 * 20;
	public static int CONNECTION_TIMEOUT = 1000 *5;
	public static String USER_AGENT = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)";
	public static String ACCEPT_LANG = "	zh-cn,en-us;q=0.8,zh-tw;q=0.5,en;q=0.3";
	public static String ACCEPT_CHARSET = "utf-8, GBK, GB2312;q=0.7,*;q=0.7";

	/*
	 * Private parameters
	 */
	private int socket_timeout = SO_TIMEOUT;
	private int connection_timeout = CONNECTION_TIMEOUT;
	private String user_agent = USER_AGENT;
	private String accept_lang = ACCEPT_LANG;
	private String accept_charset = ACCEPT_CHARSET;
	private String referer = "";

	/*
	 * Native objects
	 */
	private String charSet;
	private DefaultHttpClient client;
	private static final Map<String, String> skippedFormats = initSkippedFormats();

	public Crawler() {
		this.client = new BaseClient().getClient();
	}

	public Crawler(DefaultHttpClient client) {
		this.client = client;
	}

	/**
	 * Get method .<br>
	 * HttpClient initialization required .
	 * 
	 * @param url requested url
	 * @return content string
	 */
	public String get(String url) {
		if (pre_filtered(url)) {return null;}
		String content = getWithInterval(url, 0);
		return content;
	}

	/**
	 * Post method .<br>
	 * HttpClient initialization required .
	 * 
	 * @param url requested url
	 * @param nvpairs parameters
	 * @return content string
	 */
	public String post(String url, List<NameValuePair> nvpairs) {
		if (pre_filtered(url)) {return null;}
		String content = postWithInterval( url, nvpairs, 0);
		return content;
	}

	/**
	 * You can specify a interval time .<br>
	 * HttpClient initialization required .
	 * 
	 * @param url requested url
	 * @param millis time value
	 * @return content string
	 */
	public String getWithInterval(String url, long millis) {
		String content = null;
		// Get content
		try {
			content = doGet(client, url);
		} catch (Exception e) {
			e.printStackTrace();
		}
		
		// Spend time
		if (millis != 0) {
			try {
				Thread.sleep(millis);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
		}
		return content;
	}
	
	/**
	 * You can specify a interval time .<br>
	 * HttpClient initialization required .
	 * 
	 * @param url requested url
	 * @param millis time value
	 * @return content string
	 */
	public String postWithInterval(String url, List<NameValuePair> nvpairs,
			long millis) {
		String content = null;
		// Get content
		try {
			content = doPost(client, url, nvpairs);
		} catch (Exception e) {
			e.printStackTrace();
		}
		
		// Spend time
		if (millis != 0) {
			try {
				Thread.sleep(millis);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
		}
		return content;
	}
	
	private String doPost(DefaultHttpClient client, String url,
			List<NameValuePair> nvpairs) throws Exception {
		HttpPost httpPost = new HttpPost(url);
		httpPost.getParams().setParameter("http.socket.timeout", socket_timeout);
		httpPost.setHeader("User-Agent", user_agent);
		httpPost.setHeader("Accept-Language", accept_lang);
		httpPost.setHeader("Accept-Charset", accept_charset);
		httpPost.setHeader("Referer", referer);
		
		httpPost.setEntity(new UrlEncodedFormEntity(nvpairs, "UTF-8"));

		return requestExecutor(client, httpPost);
	}

	private String doGet(DefaultHttpClient client, String url) throws Exception {
		HttpGet httpGet = new HttpGet(url);
		httpGet.getParams().setParameter("http.socket.timeout", socket_timeout);
		httpGet.setHeader("User-Agent", user_agent);
		httpGet.setHeader("Accept-Language", accept_lang);
		httpGet.setHeader("Accept-Charset", accept_charset);
		httpGet.setHeader("Referer", referer);

		return requestExecutor(client, httpGet);
	}
	
	private String requestExecutor(DefaultHttpClient client,
			HttpRequestBase method) throws Exception {
		client.getParams().setParameter(HttpConnectionParams.SO_TIMEOUT, socket_timeout);
		client.getParams().setParameter(HttpConnectionParams.CONNECTION_TIMEOUT, connection_timeout);
		String content = null;
		HttpResponse response = client.execute(method);
		HttpEntity entity = response.getEntity();
		content = getContent(entity);

		return content;
	}

	private String getContent(HttpEntity entity) {
		if (entity == null) {
			return null;
		}

		try {
			byte[] bytes = EntityUtils.toByteArray(entity);
			charSet = EntityUtils.getContentCharSet(entity);
			entity.consumeContent();

            if (charSet == null || charSet.trim().equals("") || charSet.trim().equals("none")) {
            	String regEx = null;
            	regEx ="<(meta|\\?\\s?xml).*?(charset=|encoding=\")([\\w-]+[^'\"\\s\\?>])";
                Pattern p=Pattern.compile(regEx, Pattern.CASE_INSENSITIVE);
                Matcher m=p.matcher(new String(bytes));
                if (m.find() && m.groupCount() == 3) {
                	charSet = m.group(3);
                } else {
                	charSet = "utf-8";
                }
            }

            if (charSet != null && charSet != "") {
            	return new String(bytes, charSet);
            } else {
            	return new String(bytes);
            }

		} catch (UnsupportedEncodingException e) {
			System.out.println("ERROR - UnsupportedEncodingException: " + charSet);
		} catch (IOException e) {
			e.printStackTrace();
		} catch (Exception e) {
			e.printStackTrace();
		}

		return null;
	}
	
	public boolean pre_filtered(String url) {

		if (url == null || url.trim().equals("")) {return true;}

		String fileType = getFileTypeFromUrl(url);
		if (skippedFormats.containsKey(fileType)) {return true;}

		return false;
	}

	public String getFileTypeFromUrl(String url) {
		String type = "";
		if (url.indexOf('?') > 0) {
			url = url.substring(0, url.indexOf('?'));
		}
		String temp[] = url.split("\\.");
		if (temp.length > 1) {
			type = temp[temp.length -1].toLowerCase();
		}
		return type;
	}

	private static Map<String, String> initSkippedFormats() {
		Map<String, String> formats = new HashMap<String, String>();
		formats.put("pdf", ".pdf");
		formats.put("doc", ".doc");
		formats.put("xls", ".xls");
		formats.put("ps", ".ps");
		formats.put("ppt", ".ppt");
		formats.put("rtf", ".rtf");
		return formats;
	}

	public String getCharSet() {
		return charSet;
	}

	public int getSocket_timeout() {
		return socket_timeout;
	}

	public void setSocket_timeout(int socket_timeout) {
		this.socket_timeout = socket_timeout;
	}

	public int getConnection_timeout() {
		return connection_timeout;
	}

	public void setConnection_timeout(int connection_timeout) {
		this.connection_timeout = connection_timeout;
	}

	public String getUser_agent() {
		return user_agent;
	}

	public void setUser_agent(String user_agent) {
		this.user_agent = user_agent;
	}

	public String getAccept_lang() {
		return accept_lang;
	}

	public void setAccept_lang(String accept_lang) {
		this.accept_lang = accept_lang;
	}

	public String getAccept_charset() {
		return accept_charset;
	}

	public void setAccept_charset(String accept_charset) {
		this.accept_charset = accept_charset;
	}

	public String getReferer() {
		return referer;
	}

	public void setReferer(String referer) {
		this.referer = referer;
	}

	public List<Cookie> getCookies() {
		if (client == null) return null;
		return client.getCookieStore().getCookies();
	}

	public void setCookies(List<Cookie> cookies) {
		if (client != null) {
			client.getCookieStore().clear();
			for (Cookie cookie : cookies) {
				client.getCookieStore().addCookie(cookie);
			}
		}
	}
	
	public DefaultHttpClient getClient() {
		return client;
	}

	public void setClient(DefaultHttpClient client) {
		this.client = client;
	}

	public void setProxy(String[] proxy) {
		HttpHost httpHost = new HttpHost(proxy[0], Integer.parseInt(proxy[1]));
		client.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, httpHost);
	}

}

